{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import math\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_bn_act(in_, out_, kernel_size,\n",
    "                stride=1, groups=1, bias=True,\n",
    "                eps=1e-3, momentum=0.01):\n",
    "    return nn.Sequential(\n",
    "        SamePadConv2d(in_, out_, kernel_size, stride, groups=groups, bias=bias),\n",
    "        nn.BatchNorm2d(out_, eps, momentum),\n",
    "        Mish()\n",
    "    )\n",
    "\n",
    "\n",
    "class SamePadConv2d(nn.Conv2d):\n",
    "    \"\"\"\n",
    "    Conv with TF padding='same'\n",
    "    https://github.com/pytorch/pytorch/issues/3867#issuecomment-349279036\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True, padding_mode=\"zeros\"):\n",
    "        super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias, padding_mode)\n",
    "\n",
    "    def get_pad_odd(self, in_, weight, stride, dilation):\n",
    "        effective_filter_size_rows = (weight - 1) * dilation + 1\n",
    "        out_rows = (in_ + stride - 1) // stride\n",
    "        padding_needed = max(0, (out_rows - 1) * stride + effective_filter_size_rows - in_)\n",
    "        padding_rows = max(0, (out_rows - 1) * stride + (weight - 1) * dilation + 1 - in_)\n",
    "        rows_odd = (padding_rows % 2 != 0)\n",
    "        return padding_rows, rows_odd\n",
    "\n",
    "    def forward(self, x):\n",
    "        padding_rows, rows_odd = self.get_pad_odd(x.shape[2], self.weight.shape[2], self.stride[0], self.dilation[0])\n",
    "        padding_cols, cols_odd = self.get_pad_odd(x.shape[3], self.weight.shape[3], self.stride[1], self.dilation[1])\n",
    "\n",
    "        if rows_odd or cols_odd:\n",
    "            x = F.pad(x, [0, int(cols_odd), 0, int(rows_odd)])\n",
    "\n",
    "        return F.conv2d(x, self.weight, self.bias, self.stride,\n",
    "                        padding=(padding_rows // 2, padding_cols // 2),\n",
    "                        dilation=self.dilation, groups=self.groups)\n",
    "\n",
    "\n",
    "class Swish(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return x * torch.sigmoid(x)\n",
    "    \n",
    "class Mish(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return x * torch.tanh(F.softplus(x))\n",
    "\n",
    "class Flatten(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return x.view(x.shape[0], -1)\n",
    "\n",
    "\n",
    "class SEModule(nn.Module):\n",
    "    def __init__(self, in_, squeeze_ch):\n",
    "        super().__init__()\n",
    "        self.se = nn.Sequential(\n",
    "            nn.AdaptiveAvgPool2d(1),\n",
    "            nn.Conv2d(in_, squeeze_ch, kernel_size=1, stride=1, padding=0, bias=True),\n",
    "            Mish(),\n",
    "            nn.Conv2d(squeeze_ch, in_, kernel_size=1, stride=1, padding=0, bias=True),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x * torch.sigmoid(self.se(x))\n",
    "\n",
    "\n",
    "class DropConnect(nn.Module):\n",
    "    def __init__(self, ratio):\n",
    "        super().__init__()\n",
    "        self.ratio = 1.0 - ratio\n",
    "\n",
    "    def forward(self, x):\n",
    "        if not self.training:\n",
    "            return x\n",
    "\n",
    "        random_tensor = self.ratio\n",
    "        random_tensor += torch.rand([x.shape[0], 1, 1, 1], dtype=torch.float, device=x.device)\n",
    "        random_tensor.requires_grad_(False)\n",
    "        return x / self.ratio * random_tensor.floor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MBConv(nn.Module):\n",
    "    def __init__(self, in_, out_, expand,\n",
    "                 kernel_size, stride, skip,\n",
    "                 se_ratio, dc_ratio=0.2):\n",
    "        super().__init__()\n",
    "        mid_ = in_ * expand\n",
    "        self.expand_conv = conv_bn_act(in_, mid_, kernel_size=1, bias=False) if expand != 1 else nn.Identity()\n",
    "\n",
    "        self.depth_wise_conv = conv_bn_act(mid_, mid_,\n",
    "                                           kernel_size=kernel_size, stride=stride,\n",
    "                                           groups=mid_, bias=False)\n",
    "\n",
    "        self.se = SEModule(mid_, int(in_ * se_ratio)) if se_ratio > 0 else nn.Identity()\n",
    "\n",
    "        self.project_conv = nn.Sequential(\n",
    "            SamePadConv2d(mid_, out_, kernel_size=1, stride=1, bias=False),\n",
    "            nn.BatchNorm2d(out_, 1e-3, 0.01)\n",
    "        )\n",
    "\n",
    "        # if _block_args.id_skip:\n",
    "        # and all(s == 1 for s in self._block_args.strides)\n",
    "        # and self._block_args.input_filters == self._block_args.output_filters:\n",
    "        self.skip = skip and (stride == 1) and (in_ == out_)\n",
    "\n",
    "        # DropConnect\n",
    "        # self.dropconnect = DropConnect(dc_ratio) if dc_ratio > 0 else nn.Identity()\n",
    "        # Original TF Repo not using drop_rate\n",
    "        # https://github.com/tensorflow/tpu/blob/05f7b15cdf0ae36bac84beb4aef0a09983ce8f66/models/official/efficientnet/efficientnet_model.py#L408\n",
    "        self.dropconnect = nn.Identity()\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        expand = self.expand_conv(inputs)\n",
    "        x = self.depth_wise_conv(expand)\n",
    "        x = self.se(x)\n",
    "        x = self.project_conv(x)\n",
    "        if self.skip:\n",
    "            x = self.dropconnect(x)\n",
    "            x = x + inputs\n",
    "        return x\n",
    "\n",
    "\n",
    "class MBBlock(nn.Module):\n",
    "    def __init__(self, in_, out_, expand, kernel, stride, num_repeat, skip, se_ratio, drop_connect_ratio=0.2):\n",
    "        super().__init__()\n",
    "        layers = [MBConv(in_, out_, expand, kernel, stride, skip, se_ratio, drop_connect_ratio)]\n",
    "        for i in range(1, num_repeat):\n",
    "            layers.append(MBConv(out_, out_, expand, kernel, 1, skip, se_ratio, drop_connect_ratio))\n",
    "        self.layers = nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.layers(x)\n",
    "\n",
    "\n",
    "class EfficientNet(nn.Module):\n",
    "    def __init__(self, width_coeff, depth_coeff,\n",
    "                 depth_div=8, min_depth=None,\n",
    "                 dropout_rate=0.2, drop_connect_rate=0.2,\n",
    "                 num_classes=1000):\n",
    "        super().__init__()\n",
    "        min_depth = min_depth or depth_div\n",
    "\n",
    "        self.stem = conv_bn_act(3, 32, kernel_size=3, stride=2, bias=False)\n",
    "\n",
    "        def renew_ch(x):\n",
    "            if not width_coeff:\n",
    "                return x\n",
    "\n",
    "            x *= width_coeff\n",
    "            new_x = max(min_depth, int(x + depth_div / 2) // depth_div * depth_div)\n",
    "            if new_x < 0.9 * x:\n",
    "                new_x += depth_div\n",
    "            return int(new_x)\n",
    "\n",
    "        def renew_repeat(x):\n",
    "            return int(math.ceil(x * depth_coeff))\n",
    "\n",
    "        self.blocks = nn.Sequential(\n",
    "            #       input channel  output    expand  k  s                   skip  se\n",
    "            MBBlock(renew_ch(32), renew_ch(16), 1, 3, 1, renew_repeat(1), True, 0.25, drop_connect_rate),\n",
    "            MBBlock(renew_ch(16), renew_ch(24), 6, 3, 2, renew_repeat(2), True, 0.25, drop_connect_rate),\n",
    "            MBBlock(renew_ch(24), renew_ch(40), 6, 5, 2, renew_repeat(2), True, 0.25, drop_connect_rate),\n",
    "            MBBlock(renew_ch(40), renew_ch(80), 6, 3, 2, renew_repeat(3), True, 0.25, drop_connect_rate),\n",
    "            MBBlock(renew_ch(80), renew_ch(112), 6, 5, 1, renew_repeat(3), True, 0.25, drop_connect_rate),\n",
    "            MBBlock(renew_ch(112), renew_ch(192), 6, 5, 2, renew_repeat(4), True, 0.25, drop_connect_rate),\n",
    "            MBBlock(renew_ch(192), renew_ch(320), 6, 3, 1, renew_repeat(1), True, 0.25, drop_connect_rate)\n",
    "        )\n",
    "\n",
    "        self.head = nn.Sequential(\n",
    "            *conv_bn_act(renew_ch(320), renew_ch(1280), kernel_size=1, bias=False),\n",
    "            nn.AdaptiveAvgPool2d(1),\n",
    "            nn.Dropout2d(dropout_rate, True) if dropout_rate > 0 else nn.Identity(),\n",
    "            Flatten(),\n",
    "            nn.Linear(renew_ch(1280), num_classes)\n",
    "        )\n",
    "\n",
    "        self.init_weights()\n",
    "\n",
    "    def init_weights(self):\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, SamePadConv2d):\n",
    "                nn.init.kaiming_normal_(m.weight, mode=\"fan_out\")\n",
    "            elif isinstance(m, nn.Linear):\n",
    "                init_range = 1.0 / math.sqrt(m.weight.shape[1])\n",
    "                nn.init.uniform_(m.weight, -init_range, init_range)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        stem = self.stem(inputs)\n",
    "        x = self.blocks(stem)\n",
    "        head = self.head(x)\n",
    "        return head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:05, 28576977.63it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = EfficientNet(1, 1) # B0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 50.4402.. Average time per step: 0.1290.. Train loss: 1.7987.. Train accuracy: 0.3838.. Top-3 train accuracy: 0.7326.. Test loss: 3.1927.. Test accuracy: 0.0996.. Top-3 test accuracy: 0.2988\n",
      "Epoch 2/100.. Time per epoch: 49.3066.. Average time per step: 0.1261.. Train loss: 1.3847.. Train accuracy: 0.4960.. Top-3 train accuracy: 0.8210.. Test loss: 1.4034.. Test accuracy: 0.4967.. Top-3 test accuracy: 0.8245\n",
      "Epoch 3/100.. Time per epoch: 49.3080.. Average time per step: 0.1261.. Train loss: 1.2564.. Train accuracy: 0.5485.. Top-3 train accuracy: 0.8460.. Test loss: 1.2902.. Test accuracy: 0.5693.. Top-3 test accuracy: 0.8589\n",
      "Epoch 4/100.. Time per epoch: 49.2643.. Average time per step: 0.1260.. Train loss: 1.1729.. Train accuracy: 0.5823.. Top-3 train accuracy: 0.8648.. Test loss: 1.1777.. Test accuracy: 0.6085.. Top-3 test accuracy: 0.8756\n",
      "Epoch 5/100.. Time per epoch: 49.3920.. Average time per step: 0.1263.. Train loss: 1.0931.. Train accuracy: 0.6144.. Top-3 train accuracy: 0.8827.. Test loss: 1.0625.. Test accuracy: 0.6342.. Top-3 test accuracy: 0.8961\n",
      "Epoch 6/100.. Time per epoch: 49.0098.. Average time per step: 0.1253.. Train loss: 1.0366.. Train accuracy: 0.6336.. Top-3 train accuracy: 0.8922.. Test loss: 1.0304.. Test accuracy: 0.6474.. Top-3 test accuracy: 0.9014\n",
      "Epoch 7/100.. Time per epoch: 49.4530.. Average time per step: 0.1265.. Train loss: 0.9810.. Train accuracy: 0.6546.. Top-3 train accuracy: 0.9016.. Test loss: 1.0159.. Test accuracy: 0.6561.. Top-3 test accuracy: 0.9074\n",
      "Epoch 8/100.. Time per epoch: 49.0493.. Average time per step: 0.1254.. Train loss: 0.9459.. Train accuracy: 0.6720.. Top-3 train accuracy: 0.9080.. Test loss: 0.9873.. Test accuracy: 0.6668.. Top-3 test accuracy: 0.9168\n",
      "Epoch 9/100.. Time per epoch: 49.1067.. Average time per step: 0.1256.. Train loss: 0.9003.. Train accuracy: 0.6861.. Top-3 train accuracy: 0.9137.. Test loss: 0.9472.. Test accuracy: 0.6773.. Top-3 test accuracy: 0.9197\n",
      "Epoch 10/100.. Time per epoch: 49.2916.. Average time per step: 0.1261.. Train loss: 0.8687.. Train accuracy: 0.6959.. Top-3 train accuracy: 0.9189.. Test loss: 0.8339.. Test accuracy: 0.7165.. Top-3 test accuracy: 0.9210\n",
      "Epoch 11/100.. Time per epoch: 49.1979.. Average time per step: 0.1258.. Train loss: 0.8402.. Train accuracy: 0.7084.. Top-3 train accuracy: 0.9249.. Test loss: 0.9018.. Test accuracy: 0.7037.. Top-3 test accuracy: 0.9216\n",
      "Epoch 12/100.. Time per epoch: 49.2692.. Average time per step: 0.1260.. Train loss: 0.8175.. Train accuracy: 0.7148.. Top-3 train accuracy: 0.9276.. Test loss: 0.8124.. Test accuracy: 0.7244.. Top-3 test accuracy: 0.9299\n",
      "Epoch 13/100.. Time per epoch: 49.4204.. Average time per step: 0.1264.. Train loss: 0.7899.. Train accuracy: 0.7243.. Top-3 train accuracy: 0.9334.. Test loss: 0.8301.. Test accuracy: 0.7206.. Top-3 test accuracy: 0.9330\n",
      "Epoch 14/100.. Time per epoch: 49.3249.. Average time per step: 0.1262.. Train loss: 0.7725.. Train accuracy: 0.7310.. Top-3 train accuracy: 0.9342.. Test loss: 0.8002.. Test accuracy: 0.7179.. Top-3 test accuracy: 0.9282\n",
      "Epoch 15/100.. Time per epoch: 49.1181.. Average time per step: 0.1256.. Train loss: 0.7518.. Train accuracy: 0.7385.. Top-3 train accuracy: 0.9376.. Test loss: 0.7951.. Test accuracy: 0.7314.. Top-3 test accuracy: 0.9298\n",
      "Epoch 16/100.. Time per epoch: 49.2386.. Average time per step: 0.1259.. Train loss: 0.7350.. Train accuracy: 0.7452.. Top-3 train accuracy: 0.9409.. Test loss: 0.7603.. Test accuracy: 0.7428.. Top-3 test accuracy: 0.9388\n",
      "Epoch 17/100.. Time per epoch: 49.1444.. Average time per step: 0.1257.. Train loss: 0.7259.. Train accuracy: 0.7468.. Top-3 train accuracy: 0.9413.. Test loss: 0.7287.. Test accuracy: 0.7430.. Top-3 test accuracy: 0.9343\n",
      "Epoch 18/100.. Time per epoch: 49.3970.. Average time per step: 0.1263.. Train loss: 0.7105.. Train accuracy: 0.7514.. Top-3 train accuracy: 0.9429.. Test loss: 0.7348.. Test accuracy: 0.7446.. Top-3 test accuracy: 0.9389\n",
      "Epoch 19/100.. Time per epoch: 48.9889.. Average time per step: 0.1253.. Train loss: 0.9247.. Train accuracy: 0.6810.. Top-3 train accuracy: 0.9077.. Test loss: 0.9543.. Test accuracy: 0.6880.. Top-3 test accuracy: 0.9108\n",
      "Epoch 20/100.. Time per epoch: 49.0206.. Average time per step: 0.1254.. Train loss: 0.7698.. Train accuracy: 0.7314.. Top-3 train accuracy: 0.9364.. Test loss: 0.7729.. Test accuracy: 0.7344.. Top-3 test accuracy: 0.9380\n",
      "Epoch 21/100.. Time per epoch: 49.0019.. Average time per step: 0.1253.. Train loss: 0.7226.. Train accuracy: 0.7482.. Top-3 train accuracy: 0.9410.. Test loss: 0.7157.. Test accuracy: 0.7543.. Top-3 test accuracy: 0.9425\n",
      "Epoch 22/100.. Time per epoch: 49.1265.. Average time per step: 0.1256.. Train loss: 0.7576.. Train accuracy: 0.7377.. Top-3 train accuracy: 0.9366.. Test loss: 0.7595.. Test accuracy: 0.7404.. Top-3 test accuracy: 0.9385\n",
      "Epoch 23/100.. Time per epoch: 49.0029.. Average time per step: 0.1253.. Train loss: 0.6791.. Train accuracy: 0.7631.. Top-3 train accuracy: 0.9483.. Test loss: 0.6784.. Test accuracy: 0.7662.. Top-3 test accuracy: 0.9465\n",
      "Epoch 24/100.. Time per epoch: 49.1601.. Average time per step: 0.1257.. Train loss: 0.8025.. Train accuracy: 0.7200.. Top-3 train accuracy: 0.9263.. Test loss: 1.9134.. Test accuracy: 0.4482.. Top-3 test accuracy: 0.7673\n",
      "Epoch 25/100.. Time per epoch: 48.9001.. Average time per step: 0.1251.. Train loss: 0.8984.. Train accuracy: 0.6842.. Top-3 train accuracy: 0.9166.. Test loss: 0.7854.. Test accuracy: 0.7284.. Top-3 test accuracy: 0.9327\n",
      "Epoch 26/100.. Time per epoch: 48.9767.. Average time per step: 0.1253.. Train loss: 0.7436.. Train accuracy: 0.7381.. Top-3 train accuracy: 0.9387.. Test loss: 0.7377.. Test accuracy: 0.7492.. Top-3 test accuracy: 0.9388\n",
      "Epoch 27/100.. Time per epoch: 49.0688.. Average time per step: 0.1255.. Train loss: 0.6914.. Train accuracy: 0.7578.. Top-3 train accuracy: 0.9474.. Test loss: 0.6995.. Test accuracy: 0.7557.. Top-3 test accuracy: 0.9467\n",
      "Epoch 28/100.. Time per epoch: 49.0220.. Average time per step: 0.1254.. Train loss: 0.6570.. Train accuracy: 0.7696.. Top-3 train accuracy: 0.9516.. Test loss: 0.6706.. Test accuracy: 0.7690.. Top-3 test accuracy: 0.9466\n",
      "Epoch 29/100.. Time per epoch: 49.0072.. Average time per step: 0.1253.. Train loss: 0.6397.. Train accuracy: 0.7753.. Top-3 train accuracy: 0.9525.. Test loss: 0.6848.. Test accuracy: 0.7669.. Top-3 test accuracy: 0.9438\n",
      "Epoch 30/100.. Time per epoch: 48.6563.. Average time per step: 0.1244.. Train loss: 0.6202.. Train accuracy: 0.7830.. Top-3 train accuracy: 0.9544.. Test loss: 0.6848.. Test accuracy: 0.7631.. Top-3 test accuracy: 0.9466\n",
      "Epoch 31/100.. Time per epoch: 48.5765.. Average time per step: 0.1242.. Train loss: 0.6386.. Train accuracy: 0.7795.. Top-3 train accuracy: 0.9516.. Test loss: 0.8317.. Test accuracy: 0.7156.. Top-3 test accuracy: 0.9232\n",
      "Epoch 32/100.. Time per epoch: 48.5999.. Average time per step: 0.1243.. Train loss: 0.6353.. Train accuracy: 0.7774.. Top-3 train accuracy: 0.9541.. Test loss: 0.6486.. Test accuracy: 0.7819.. Top-3 test accuracy: 0.9498\n",
      "Epoch 33/100.. Time per epoch: 48.8238.. Average time per step: 0.1249.. Train loss: 0.5960.. Train accuracy: 0.7904.. Top-3 train accuracy: 0.9590.. Test loss: 0.6516.. Test accuracy: 0.7830.. Top-3 test accuracy: 0.9500\n",
      "Epoch 34/100.. Time per epoch: 48.9168.. Average time per step: 0.1251.. Train loss: 0.5846.. Train accuracy: 0.7961.. Top-3 train accuracy: 0.9598.. Test loss: 0.6527.. Test accuracy: 0.7760.. Top-3 test accuracy: 0.9499\n",
      "Epoch 35/100.. Time per epoch: 48.9424.. Average time per step: 0.1252.. Train loss: 0.5698.. Train accuracy: 0.8002.. Top-3 train accuracy: 0.9605.. Test loss: 0.6477.. Test accuracy: 0.7778.. Top-3 test accuracy: 0.9515\n",
      "Epoch 36/100.. Time per epoch: 48.8026.. Average time per step: 0.1248.. Train loss: 0.5679.. Train accuracy: 0.8000.. Top-3 train accuracy: 0.9617.. Test loss: 0.6766.. Test accuracy: 0.7734.. Top-3 test accuracy: 0.9474\n",
      "Epoch 37/100.. Time per epoch: 48.6493.. Average time per step: 0.1244.. Train loss: 0.6120.. Train accuracy: 0.7876.. Top-3 train accuracy: 0.9557.. Test loss: 0.7340.. Test accuracy: 0.7713.. Top-3 test accuracy: 0.9429\n",
      "Epoch 38/100.. Time per epoch: 48.6430.. Average time per step: 0.1244.. Train loss: 0.5621.. Train accuracy: 0.8045.. Top-3 train accuracy: 0.9623.. Test loss: 0.6511.. Test accuracy: 0.7760.. Top-3 test accuracy: 0.9502\n",
      "Epoch 39/100.. Time per epoch: 48.8297.. Average time per step: 0.1249.. Train loss: 0.5611.. Train accuracy: 0.8046.. Top-3 train accuracy: 0.9625.. Test loss: 0.6580.. Test accuracy: 0.7771.. Top-3 test accuracy: 0.9514\n",
      "Epoch 40/100.. Time per epoch: 48.9674.. Average time per step: 0.1252.. Train loss: 0.5342.. Train accuracy: 0.8137.. Top-3 train accuracy: 0.9666.. Test loss: 0.6357.. Test accuracy: 0.7793.. Top-3 test accuracy: 0.9510\n",
      "Epoch 41/100.. Time per epoch: 50.4555.. Average time per step: 0.1290.. Train loss: 0.5323.. Train accuracy: 0.8150.. Top-3 train accuracy: 0.9653.. Test loss: 0.6577.. Test accuracy: 0.7778.. Top-3 test accuracy: 0.9493\n",
      "Epoch 42/100.. Time per epoch: 49.4627.. Average time per step: 0.1265.. Train loss: 0.5549.. Train accuracy: 0.8075.. Top-3 train accuracy: 0.9633.. Test loss: 0.6247.. Test accuracy: 0.7892.. Top-3 test accuracy: 0.9505\n",
      "Epoch 43/100.. Time per epoch: 50.0908.. Average time per step: 0.1281.. Train loss: 0.5132.. Train accuracy: 0.8186.. Top-3 train accuracy: 0.9684.. Test loss: 0.6250.. Test accuracy: 0.7889.. Top-3 test accuracy: 0.9554\n",
      "Epoch 44/100.. Time per epoch: 50.0182.. Average time per step: 0.1279.. Train loss: 0.5112.. Train accuracy: 0.8205.. Top-3 train accuracy: 0.9681.. Test loss: 0.6466.. Test accuracy: 0.7843.. Top-3 test accuracy: 0.9512\n",
      "Epoch 45/100.. Time per epoch: 50.2639.. Average time per step: 0.1286.. Train loss: 0.4899.. Train accuracy: 0.8272.. Top-3 train accuracy: 0.9704.. Test loss: 0.6275.. Test accuracy: 0.7940.. Top-3 test accuracy: 0.9537\n",
      "Epoch 46/100.. Time per epoch: 50.4212.. Average time per step: 0.1290.. Train loss: 0.5344.. Train accuracy: 0.8137.. Top-3 train accuracy: 0.9666.. Test loss: 0.6605.. Test accuracy: 0.7773.. Top-3 test accuracy: 0.9522\n",
      "Epoch 47/100.. Time per epoch: 50.4945.. Average time per step: 0.1291.. Train loss: 0.4947.. Train accuracy: 0.8260.. Top-3 train accuracy: 0.9695.. Test loss: 0.6437.. Test accuracy: 0.7877.. Top-3 test accuracy: 0.9533\n",
      "Epoch 48/100.. Time per epoch: 50.4131.. Average time per step: 0.1289.. Train loss: 0.4726.. Train accuracy: 0.8339.. Top-3 train accuracy: 0.9724.. Test loss: 0.6158.. Test accuracy: 0.7967.. Top-3 test accuracy: 0.9558\n",
      "Epoch 49/100.. Time per epoch: 50.4497.. Average time per step: 0.1290.. Train loss: 0.4700.. Train accuracy: 0.8359.. Top-3 train accuracy: 0.9716.. Test loss: 0.5994.. Test accuracy: 0.7963.. Top-3 test accuracy: 0.9593\n",
      "Epoch 50/100.. Time per epoch: 50.4373.. Average time per step: 0.1290.. Train loss: 0.4653.. Train accuracy: 0.8360.. Top-3 train accuracy: 0.9732.. Test loss: 0.6603.. Test accuracy: 0.7893.. Top-3 test accuracy: 0.9514\n",
      "Epoch 51/100.. Time per epoch: 50.4910.. Average time per step: 0.1291.. Train loss: 0.4603.. Train accuracy: 0.8368.. Top-3 train accuracy: 0.9736.. Test loss: 0.6104.. Test accuracy: 0.8000.. Top-3 test accuracy: 0.9579\n",
      "Epoch 52/100.. Time per epoch: 50.5775.. Average time per step: 0.1294.. Train loss: 0.4587.. Train accuracy: 0.8387.. Top-3 train accuracy: 0.9743.. Test loss: 0.6089.. Test accuracy: 0.8012.. Top-3 test accuracy: 0.9586\n",
      "Epoch 53/100.. Time per epoch: 50.4548.. Average time per step: 0.1290.. Train loss: 0.4480.. Train accuracy: 0.8430.. Top-3 train accuracy: 0.9750.. Test loss: 0.6165.. Test accuracy: 0.7977.. Top-3 test accuracy: 0.9524\n",
      "Epoch 54/100.. Time per epoch: 50.3500.. Average time per step: 0.1288.. Train loss: 0.6175.. Train accuracy: 0.7869.. Top-3 train accuracy: 0.9537.. Test loss: 0.8966.. Test accuracy: 0.7021.. Top-3 test accuracy: 0.9247\n",
      "Epoch 55/100.. Time per epoch: 50.2999.. Average time per step: 0.1286.. Train loss: 0.6152.. Train accuracy: 0.7839.. Top-3 train accuracy: 0.9577.. Test loss: 0.6420.. Test accuracy: 0.7845.. Top-3 test accuracy: 0.9510\n",
      "Epoch 56/100.. Time per epoch: 50.6236.. Average time per step: 0.1295.. Train loss: 0.4790.. Train accuracy: 0.8317.. Top-3 train accuracy: 0.9726.. Test loss: 0.6318.. Test accuracy: 0.7917.. Top-3 test accuracy: 0.9564\n",
      "Epoch 57/100.. Time per epoch: 50.6335.. Average time per step: 0.1295.. Train loss: 0.4415.. Train accuracy: 0.8453.. Top-3 train accuracy: 0.9763.. Test loss: 0.6153.. Test accuracy: 0.7991.. Top-3 test accuracy: 0.9575\n",
      "Epoch 58/100.. Time per epoch: 50.5278.. Average time per step: 0.1292.. Train loss: 0.4448.. Train accuracy: 0.8437.. Top-3 train accuracy: 0.9763.. Test loss: 0.8135.. Test accuracy: 0.7840.. Top-3 test accuracy: 0.9509\n",
      "Epoch 59/100.. Time per epoch: 50.2828.. Average time per step: 0.1286.. Train loss: 0.4571.. Train accuracy: 0.8392.. Top-3 train accuracy: 0.9752.. Test loss: 0.6530.. Test accuracy: 0.7943.. Top-3 test accuracy: 0.9536\n",
      "Epoch 60/100.. Time per epoch: 50.4545.. Average time per step: 0.1290.. Train loss: 0.4262.. Train accuracy: 0.8516.. Top-3 train accuracy: 0.9769.. Test loss: 0.6297.. Test accuracy: 0.8006.. Top-3 test accuracy: 0.9535\n",
      "Epoch 61/100.. Time per epoch: 49.9934.. Average time per step: 0.1279.. Train loss: 0.3995.. Train accuracy: 0.8600.. Top-3 train accuracy: 0.9798.. Test loss: 0.6314.. Test accuracy: 0.7931.. Top-3 test accuracy: 0.9557\n",
      "Epoch 62/100.. Time per epoch: 50.4537.. Average time per step: 0.1290.. Train loss: 0.3942.. Train accuracy: 0.8605.. Top-3 train accuracy: 0.9805.. Test loss: 0.6175.. Test accuracy: 0.7995.. Top-3 test accuracy: 0.9555\n",
      "Epoch 63/100.. Time per epoch: 50.4708.. Average time per step: 0.1291.. Train loss: 0.3899.. Train accuracy: 0.8618.. Top-3 train accuracy: 0.9798.. Test loss: 0.6348.. Test accuracy: 0.7975.. Top-3 test accuracy: 0.9554\n",
      "Epoch 64/100.. Time per epoch: 50.2779.. Average time per step: 0.1286.. Train loss: 0.5134.. Train accuracy: 0.8219.. Top-3 train accuracy: 0.9678.. Test loss: 0.6433.. Test accuracy: 0.7841.. Top-3 test accuracy: 0.9538\n",
      "Epoch 65/100.. Time per epoch: 50.3086.. Average time per step: 0.1287.. Train loss: 0.4580.. Train accuracy: 0.8379.. Top-3 train accuracy: 0.9745.. Test loss: 0.6136.. Test accuracy: 0.7984.. Top-3 test accuracy: 0.9574\n",
      "Epoch 66/100.. Time per epoch: 50.6077.. Average time per step: 0.1294.. Train loss: 0.4016.. Train accuracy: 0.8595.. Top-3 train accuracy: 0.9802.. Test loss: 0.6445.. Test accuracy: 0.8005.. Top-3 test accuracy: 0.9571\n",
      "Epoch 67/100.. Time per epoch: 52.0633.. Average time per step: 0.1332.. Train loss: 0.4019.. Train accuracy: 0.8591.. Top-3 train accuracy: 0.9800.. Test loss: 0.6083.. Test accuracy: 0.8043.. Top-3 test accuracy: 0.9596\n",
      "Epoch 68/100.. Time per epoch: 51.9672.. Average time per step: 0.1329.. Train loss: 0.3722.. Train accuracy: 0.8680.. Top-3 train accuracy: 0.9816.. Test loss: 0.6390.. Test accuracy: 0.8040.. Top-3 test accuracy: 0.9562\n",
      "Epoch 69/100.. Time per epoch: 51.8872.. Average time per step: 0.1327.. Train loss: 0.4723.. Train accuracy: 0.8373.. Top-3 train accuracy: 0.9739.. Test loss: 0.6678.. Test accuracy: 0.7879.. Top-3 test accuracy: 0.9494\n",
      "Epoch 70/100.. Time per epoch: 51.7388.. Average time per step: 0.1323.. Train loss: 0.3999.. Train accuracy: 0.8582.. Top-3 train accuracy: 0.9810.. Test loss: 0.6132.. Test accuracy: 0.8051.. Top-3 test accuracy: 0.9594\n",
      "Epoch 71/100.. Time per epoch: 51.3905.. Average time per step: 0.1314.. Train loss: 0.3627.. Train accuracy: 0.8712.. Top-3 train accuracy: 0.9832.. Test loss: 0.6314.. Test accuracy: 0.8004.. Top-3 test accuracy: 0.9575\n",
      "Epoch 72/100.. Time per epoch: 50.0836.. Average time per step: 0.1281.. Train loss: 0.4290.. Train accuracy: 0.8490.. Top-3 train accuracy: 0.9766.. Test loss: 1.1245.. Test accuracy: 0.7283.. Top-3 test accuracy: 0.9248\n",
      "Epoch 73/100.. Time per epoch: 50.1724.. Average time per step: 0.1283.. Train loss: 0.6764.. Train accuracy: 0.7653.. Top-3 train accuracy: 0.9497.. Test loss: 0.6643.. Test accuracy: 0.7777.. Top-3 test accuracy: 0.9488\n",
      "Epoch 74/100.. Time per epoch: 50.0465.. Average time per step: 0.1280.. Train loss: 0.4531.. Train accuracy: 0.8401.. Top-3 train accuracy: 0.9745.. Test loss: 0.6199.. Test accuracy: 0.8035.. Top-3 test accuracy: 0.9544\n",
      "Epoch 75/100.. Time per epoch: 50.1331.. Average time per step: 0.1282.. Train loss: 0.3855.. Train accuracy: 0.8642.. Top-3 train accuracy: 0.9815.. Test loss: 0.6184.. Test accuracy: 0.8072.. Top-3 test accuracy: 0.9587\n",
      "Epoch 76/100.. Time per epoch: 49.9703.. Average time per step: 0.1278.. Train loss: 0.3620.. Train accuracy: 0.8727.. Top-3 train accuracy: 0.9840.. Test loss: 0.6218.. Test accuracy: 0.8069.. Top-3 test accuracy: 0.9590\n",
      "Epoch 77/100.. Time per epoch: 50.0130.. Average time per step: 0.1279.. Train loss: 0.5680.. Train accuracy: 0.8040.. Top-3 train accuracy: 0.9610.. Test loss: 0.7460.. Test accuracy: 0.7580.. Top-3 test accuracy: 0.9410\n",
      "Epoch 78/100.. Time per epoch: 50.0081.. Average time per step: 0.1279.. Train loss: 0.5067.. Train accuracy: 0.8214.. Top-3 train accuracy: 0.9692.. Test loss: 0.6223.. Test accuracy: 0.7968.. Top-3 test accuracy: 0.9570\n",
      "Epoch 79/100.. Time per epoch: 50.1944.. Average time per step: 0.1284.. Train loss: 0.5475.. Train accuracy: 0.8107.. Top-3 train accuracy: 0.9626.. Test loss: 0.7044.. Test accuracy: 0.7704.. Top-3 test accuracy: 0.9497\n",
      "Epoch 80/100.. Time per epoch: 50.0171.. Average time per step: 0.1279.. Train loss: 0.6222.. Train accuracy: 0.7819.. Top-3 train accuracy: 0.9556.. Test loss: 0.8624.. Test accuracy: 0.7806.. Top-3 test accuracy: 0.9508\n",
      "Epoch 81/100.. Time per epoch: 50.0180.. Average time per step: 0.1279.. Train loss: 0.4912.. Train accuracy: 0.8286.. Top-3 train accuracy: 0.9703.. Test loss: 0.9767.. Test accuracy: 0.7811.. Top-3 test accuracy: 0.9481\n",
      "Epoch 82/100.. Time per epoch: 49.5873.. Average time per step: 0.1268.. Train loss: 0.5530.. Train accuracy: 0.8063.. Top-3 train accuracy: 0.9633.. Test loss: 0.6439.. Test accuracy: 0.7852.. Top-3 test accuracy: 0.9481\n",
      "Epoch 83/100.. Time per epoch: 49.4120.. Average time per step: 0.1264.. Train loss: 0.4524.. Train accuracy: 0.8403.. Top-3 train accuracy: 0.9739.. Test loss: 0.6259.. Test accuracy: 0.8032.. Top-3 test accuracy: 0.9576\n",
      "Epoch 84/100.. Time per epoch: 49.6484.. Average time per step: 0.1270.. Train loss: 0.4281.. Train accuracy: 0.8486.. Top-3 train accuracy: 0.9772.. Test loss: 0.6354.. Test accuracy: 0.7977.. Top-3 test accuracy: 0.9570\n",
      "Epoch 85/100.. Time per epoch: 49.4675.. Average time per step: 0.1265.. Train loss: 0.4625.. Train accuracy: 0.8375.. Top-3 train accuracy: 0.9734.. Test loss: 0.6061.. Test accuracy: 0.8082.. Top-3 test accuracy: 0.9576\n",
      "Epoch 86/100.. Time per epoch: 49.3915.. Average time per step: 0.1263.. Train loss: 0.3836.. Train accuracy: 0.8648.. Top-3 train accuracy: 0.9811.. Test loss: 0.6543.. Test accuracy: 0.7954.. Top-3 test accuracy: 0.9534\n",
      "Epoch 87/100.. Time per epoch: 49.3665.. Average time per step: 0.1263.. Train loss: 0.4911.. Train accuracy: 0.8274.. Top-3 train accuracy: 0.9695.. Test loss: 0.6736.. Test accuracy: 0.7813.. Top-3 test accuracy: 0.9481\n",
      "Epoch 88/100.. Time per epoch: 49.3098.. Average time per step: 0.1261.. Train loss: 0.7159.. Train accuracy: 0.7535.. Top-3 train accuracy: 0.9387.. Test loss: 0.7979.. Test accuracy: 0.7394.. Top-3 test accuracy: 0.9320\n",
      "Epoch 89/100.. Time per epoch: 49.2622.. Average time per step: 0.1260.. Train loss: 0.7197.. Train accuracy: 0.7488.. Top-3 train accuracy: 0.9433.. Test loss: 1.8009.. Test accuracy: 0.7252.. Top-3 test accuracy: 0.9140\n",
      "Epoch 90/100.. Time per epoch: 49.4202.. Average time per step: 0.1264.. Train loss: 0.5988.. Train accuracy: 0.7903.. Top-3 train accuracy: 0.9581.. Test loss: 2.3361.. Test accuracy: 0.7790.. Top-3 test accuracy: 0.9492\n",
      "Epoch 91/100.. Time per epoch: 49.0772.. Average time per step: 0.1255.. Train loss: 0.5137.. Train accuracy: 0.8195.. Top-3 train accuracy: 0.9682.. Test loss: 7.5299.. Test accuracy: 0.7914.. Top-3 test accuracy: 0.9493\n",
      "Epoch 92/100.. Time per epoch: 49.0406.. Average time per step: 0.1254.. Train loss: 0.4609.. Train accuracy: 0.8369.. Top-3 train accuracy: 0.9752.. Test loss: 4.0251.. Test accuracy: 0.7927.. Top-3 test accuracy: 0.9489\n",
      "Epoch 93/100.. Time per epoch: 48.8856.. Average time per step: 0.1250.. Train loss: 0.4173.. Train accuracy: 0.8533.. Top-3 train accuracy: 0.9779.. Test loss: 0.7004.. Test accuracy: 0.7841.. Top-3 test accuracy: 0.9513\n",
      "Epoch 94/100.. Time per epoch: 48.8745.. Average time per step: 0.1250.. Train loss: 0.4079.. Train accuracy: 0.8577.. Top-3 train accuracy: 0.9785.. Test loss: 1.0587.. Test accuracy: 0.7970.. Top-3 test accuracy: 0.9511\n",
      "Epoch 95/100.. Time per epoch: 49.2131.. Average time per step: 0.1259.. Train loss: 0.4025.. Train accuracy: 0.8579.. Top-3 train accuracy: 0.9795.. Test loss: 0.6833.. Test accuracy: 0.7986.. Top-3 test accuracy: 0.9542\n",
      "Epoch 96/100.. Time per epoch: 49.2116.. Average time per step: 0.1259.. Train loss: 0.5008.. Train accuracy: 0.8261.. Top-3 train accuracy: 0.9687.. Test loss: 0.6005.. Test accuracy: 0.8053.. Top-3 test accuracy: 0.9584\n",
      "Epoch 97/100.. Time per epoch: 48.8631.. Average time per step: 0.1250.. Train loss: 0.4018.. Train accuracy: 0.8584.. Top-3 train accuracy: 0.9806.. Test loss: 1.0603.. Test accuracy: 0.7995.. Top-3 test accuracy: 0.9592\n",
      "Epoch 98/100.. Time per epoch: 48.7496.. Average time per step: 0.1247.. Train loss: 0.3603.. Train accuracy: 0.8726.. Top-3 train accuracy: 0.9839.. Test loss: 1.2199.. Test accuracy: 0.8053.. Top-3 test accuracy: 0.9586\n",
      "Epoch 99/100.. Time per epoch: 48.7731.. Average time per step: 0.1247.. Train loss: 0.3444.. Train accuracy: 0.8759.. Top-3 train accuracy: 0.9848.. Test loss: 1.0366.. Test accuracy: 0.8108.. Top-3 test accuracy: 0.9580\n",
      "Epoch 100/100.. Time per epoch: 48.7852.. Average time per step: 0.1248.. Train loss: 0.3285.. Train accuracy: 0.8830.. Top-3 train accuracy: 0.9858.. Test loss: 0.6362.. Test accuracy: 0.8074.. Top-3 test accuracy: 0.9605\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_EfficientNet_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
